<!DOCTYPE html>
<html lang="zh-CN">
    <head>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <meta name="robots" content="noodp" />
        <meta http-equiv="X-UA-Compatible" content="IE=edge, chrome=1">
        <title>14_K8S_监控实战-部署prometheus - 山脚下的脚下山</title><meta name="Description" content="转载，原为老男孩教育视频内容"><meta property="og:title" content="14_K8S_监控实战-部署prometheus" />
<meta property="og:description" content="转载，原为老男孩教育视频内容" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://scemsjyd.com/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus/" /><meta property="og:image" content="https://scemsjyd.com/logo.png"/><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2020-10-01T15:58:21+08:00" />
<meta property="article:modified_time" content="2020-10-01T15:58:21+08:00" />

<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="https://scemsjyd.com/logo.png"/>

<meta name="twitter:title" content="14_K8S_监控实战-部署prometheus"/>
<meta name="twitter:description" content="转载，原为老男孩教育视频内容"/>
<meta name="application-name" content="山脚下の脚下山">
<meta name="apple-mobile-web-app-title" content="山脚下の脚下山"><meta name="theme-color" content="#ffffff"><meta name="msapplication-TileColor" content="#da532c"><link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" />
        <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
        <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5"><link rel="manifest" href="/site.webmanifest"><link rel="canonical" href="https://scemsjyd.com/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus/" /><link rel="prev" href="https://scemsjyd.com/15_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-grafana%E5%87%BA%E5%9B%BE_alert%E5%91%8A%E8%AD%A6/" /><link rel="next" href="https://scemsjyd.com/13_k8s_%E9%85%8D%E7%BD%AE%E4%B8%AD%E5%BF%83%E5%AE%9E%E6%88%98-%E5%A4%9A%E7%8E%AF%E5%A2%83%E4%BA%A4%E4%BB%98apollo%E4%B8%89%E7%BB%84%E4%BB%B6/" /><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/normalize.css@8.0.1/normalize.min.css"><link rel="stylesheet" href="/css/style.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.13.0/css/all.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/animate.css@3.7.2/animate.min.css"><script type="application/ld+json">
    {
        "@context": "http://schema.org",
        "@type": "BlogPosting",
        "headline": "14_K8S_监控实战-部署prometheus",
        "inLanguage": "zh-CN",
        "mainEntityOfPage": {
            "@type": "WebPage",
            "@id": "https:\/\/scemsjyd.com\/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus\/"
        },"image": ["https:\/\/scemsjyd.com\/images\/Apple-Devices-Preview.png"],"genre": "posts","keywords": "K8S, 转载","wordcount":  5806 ,
        "url": "https:\/\/scemsjyd.com\/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus\/","datePublished": "2020-10-01T15:58:21+08:00","dateModified": "2020-10-01T15:58:21+08:00","license": "This work is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License.","publisher": {
            "@type": "Organization",
            "name": "Adam.Jin","logo": "https:\/\/p.qlogo.cn\/bizmail\/XpHAlE90tNCBxpvcMjtmdG52qxW6TFrUQ3VbWchBMibqlYJup0yhBzQ\/"},"author": {
                "@type": "Person",
                "name": "老男孩"
            },"description": "转载，原为老男孩教育视频内容"
    }
    </script></head>
    <body header-desktop="fixed" header-mobile="auto"><script type="text/javascript">(window.localStorage && localStorage.getItem('theme') ? localStorage.getItem('theme') === 'dark' : ('auto' === 'auto' ? window.matchMedia('(prefers-color-scheme: dark)').matches : 'auto' === 'dark')) && document.body.setAttribute('theme', 'dark');</script>

        <div id="mask"></div><div class="wrapper"><header class="desktop" id="header-desktop">
    <div class="header-wrapper">
        <div class="header-title">
            <a href="/" title="山脚下的脚下山"><span class="header-title-pre"><i class='far fa-kiss-wink-heart fa-fw'></i></span>山脚下の脚下山</a>
        </div>
        <div class="menu">
            <div class="menu-inner"><a class="menu-item" href="/"> 主页 </a><a class="menu-item" href="/posts/"> 文章 </a><a class="menu-item" href="/tags/"> 标签 </a><a class="menu-item" href="/categories/"> 分类 </a><a class="menu-item" href="/about/"> 关于 </a><a class="menu-item" href="https://github.com/scemsjyd" title="GitHub" rel="noopener noreffer" target="_blank"><i class='fab fa-github fa-fw'></i>  </a><span class="menu-item delimiter"></span><span class="menu-item search" id="search-desktop">
                        <input type="text" placeholder="搜索文章标题或内容..." id="search-input-desktop">
                        <a href="javascript:void(0);" class="search-button search-toggle" id="search-toggle-desktop" title="搜索">
                            <i class="fas fa-search fa-fw"></i>
                        </a>
                        <a href="javascript:void(0);" class="search-button search-clear" id="search-clear-desktop" title="清空">
                            <i class="fas fa-times-circle fa-fw"></i>
                        </a>
                        <span class="search-button search-loading" id="search-loading-desktop">
                            <i class="fas fa-spinner fa-fw fa-spin"></i>
                        </span>
                    </span><a href="javascript:void(0);" class="menu-item theme-switch" title="切换主题">
                    <i class="fas fa-adjust fa-fw"></i>
                </a>
            </div>
        </div>
    </div>
</header><header class="mobile" id="header-mobile">
    <div class="header-container">
        <div class="header-wrapper">
            <div class="header-title">
                <a href="/" title="山脚下的脚下山"><span class="header-title-pre"><i class='far fa-kiss-wink-heart fa-fw'></i></span>山脚下の脚下山</a>
            </div>
            <div class="menu-toggle" id="menu-toggle-mobile">
                <span></span><span></span><span></span>
            </div>
        </div>
        <div class="menu" id="menu-mobile"><div class="search-wrapper">
                    <div class="search mobile" id="search-mobile">
                        <input type="text" placeholder="搜索文章标题或内容..." id="search-input-mobile">
                        <a href="javascript:void(0);" class="search-button search-toggle" id="search-toggle-mobile" title="搜索">
                            <i class="fas fa-search fa-fw"></i>
                        </a>
                        <a href="javascript:void(0);" class="search-button search-clear" id="search-clear-mobile" title="清空">
                            <i class="fas fa-times-circle fa-fw"></i>
                        </a>
                        <span class="search-button search-loading" id="search-loading-mobile">
                            <i class="fas fa-spinner fa-fw fa-spin"></i>
                        </span>
                    </div>
                    <a href="javascript:void(0);" class="search-cancel" id="search-cancel-mobile">
                        取消
                    </a>
                </div><a class="menu-item" href="/" title="">主页</a><a class="menu-item" href="/posts/" title="">文章</a><a class="menu-item" href="/tags/" title="">标签</a><a class="menu-item" href="/categories/" title="">分类</a><a class="menu-item" href="/about/" title="">关于</a><a class="menu-item" href="https://github.com/scemsjyd" title="GitHub" rel="noopener noreffer" target="_blank"><i class='fab fa-github fa-fw'></i></a><a href="javascript:void(0);" class="menu-item theme-switch" title="切换主题">
                <i class="fas fa-adjust fa-fw"></i>
            </a></div>
    </div>
</header>
<div class="search-dropdown desktop">
    <div id="search-dropdown-desktop"></div>
</div>
<div class="search-dropdown mobile">
    <div id="search-dropdown-mobile"></div>
</div>
<main class="main">
                <div class="container"><div class="toc" id="toc-auto">
            <h2 class="toc-title">目录</h2>
            <div class="toc-content" id="toc-content-auto"></div>
        </div><article class="page single"><h1 class="single-title animated flipInX">14_K8S_监控实战-部署prometheus</h1><div class="post-meta">
            <div class="post-meta-line"><span class="post-author"><a href="https://space.bilibili.com/394449264" title="Author" target="_blank" rel="noopener noreffer author" class="author"><i class="fas fa-user-circle fa-fw"></i>老男孩</a></span>&nbsp;<span class="post-category">收录于 <a href="/categories/%E8%BD%AC%E8%BD%BD/"><i class="far fa-folder fa-fw"></i>转载</a>&nbsp;<a href="/categories/k8s/"><i class="far fa-folder fa-fw"></i>K8S</a></span></div>
            <div class="post-meta-line"><i class="far fa-calendar-alt fa-fw"></i>&nbsp;<time datetime="2020-10-01">2020-10-01</time>&nbsp;<i class="fas fa-pencil-alt fa-fw"></i>&nbsp;约 5806 字&nbsp;
                <i class="far fa-clock fa-fw"></i>&nbsp;预计阅读 12 分钟&nbsp;<span id="/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus/" class="leancloud_visitors" data-flag-title="14_K8S_监控实战-部署prometheus">
                        <i class="far fa-eye fa-fw"></i>&nbsp;<span class=leancloud-visitors-count></span>&nbsp;次阅读
                    </span>&nbsp;</div>
        </div><div class="details toc" id="toc-static"  kept="">
                <div class="details-summary toc-title">
                    <span>目录</span>
                    <span><i class="details-icon fas fa-angle-right"></i></span>
                </div>
                <div class="details-content toc-content" id="toc-content-static"><nav id="TableOfContents">
  <ul>
    <li><a href="#1-prometheus前言相关">1 prometheus前言相关</a>
      <ul>
        <li><a href="#11-prometheus的特点">1.1 Prometheus的特点</a></li>
        <li><a href="#12-基本原理">1.2 基本原理</a>
          <ul>
            <li><a href="#121-原理说明">1.2.1 原理说明</a></li>
            <li><a href="#122-架构图">1.2.2 架构图:</a></li>
            <li><a href="#123-三大套件">1.2.3 三大套件</a></li>
            <li><a href="#124-架构服务过程">1.2.4 架构服务过程</a></li>
            <li><a href="#125-常用的exporter">1.2.5 常用的exporter</a></li>
          </ul>
        </li>
      </ul>
    </li>
    <li><a href="#2-部署4个exporter">2 部署4个exporter</a>
      <ul>
        <li><a href="#21-部署kube-state-metrics">2.1 部署kube-state-metrics</a>
          <ul>
            <li><a href="#211-准备docker镜像">2.1.1 准备docker镜像</a></li>
            <li><a href="#212-准备rbac资源清单">2.1.2 准备rbac资源清单</a></li>
            <li><a href="#213-准备dp资源清单">2.1.3 准备Dp资源清单</a></li>
            <li><a href="#214-应用资源配置清单">2.1.4 应用资源配置清单</a></li>
          </ul>
        </li>
        <li><a href="#22-部署node-exporter">2.2 部署node-exporter</a>
          <ul>
            <li><a href="#221-准备docker镜像">2.2.1 准备docker镜像</a></li>
            <li><a href="#222-准备ds资源清单">2.2.2 准备ds资源清单</a></li>
            <li><a href="#223-应用资源配置清单">2.2.3 应用资源配置清单：</a></li>
          </ul>
        </li>
        <li><a href="#23-部署cadvisor">2.3 部署cadvisor</a>
          <ul>
            <li><a href="#231-准备docker镜像">2.3.1 准备docker镜像</a></li>
            <li><a href="#232-准备ds资源清单">2.3.2 准备ds资源清单</a></li>
            <li><a href="#233-应用资源配置清单">2.3.3 应用资源配置清单：</a></li>
          </ul>
        </li>
        <li><a href="#24-部署blackbox-exporter">2.4 部署blackbox-exporter</a>
          <ul>
            <li><a href="#241-准备docker镜像">2.4.1 准备docker镜像</a></li>
            <li><a href="#242-准备cm资源清单">2.4.2 准备cm资源清单</a></li>
            <li><a href="#243-准备dp资源清单">2.4.3 准备dp资源清单</a></li>
            <li><a href="#244-准备svc资源清单">2.4.4 准备svc资源清单</a></li>
            <li><a href="#245-准备ingress资源清单">2.4.5 准备ingress资源清单</a></li>
            <li><a href="#246-添加域名解析">2.4.6 添加域名解析</a></li>
            <li><a href="#247-应用资源配置清单">2.4.7 应用资源配置清单</a></li>
            <li><a href="#248-访问域名测试">2.4.8 访问域名测试</a></li>
          </ul>
        </li>
      </ul>
    </li>
    <li><a href="#3-部署prometheus-server">3 部署prometheus server</a>
      <ul>
        <li><a href="#31-准备prometheus-server环境">3.1 准备prometheus server环境</a>
          <ul>
            <li><a href="#311-准备docker镜像">3.1.1 准备docker镜像</a></li>
            <li><a href="#312-准备rbac资源清单">3.1.2 准备rbac资源清单</a></li>
            <li><a href="#313-准备dp资源清单">3.1.3 准备dp资源清单</a></li>
            <li><a href="#314-准备svc资源清单">3.1.4 准备svc资源清单</a></li>
            <li><a href="#315-准备ingress资源清单">3.1.5 准备ingress资源清单</a></li>
            <li><a href="#316-添加域名解析">3.1.6 添加域名解析</a></li>
          </ul>
        </li>
        <li><a href="#32-部署prometheus-server">3.2 部署prometheus server</a>
          <ul>
            <li><a href="#321-准备目录和证书">3.2.1 准备目录和证书</a></li>
            <li><a href="#322-创建prometheus配置文件">3.2.2 创建prometheus配置文件</a></li>
            <li><a href="#323-应用资源配置清单">3.2.3 应用资源配置清单</a></li>
            <li><a href="#324-浏览器验证">3.2.4 浏览器验证</a></li>
          </ul>
        </li>
      </ul>
    </li>
    <li><a href="#4-使服务能被prometheus自动监控">4 使服务能被prometheus自动监控</a>
      <ul>
        <li><a href="#41-让traefik能被自动监控">4.1 让traefik能被自动监控</a>
          <ul>
            <li><a href="#411-修改traefik的yaml">4.1.1 修改traefik的yaml</a></li>
            <li><a href="#412-应用配置查看">4.1.2 应用配置查看</a></li>
          </ul>
        </li>
        <li><a href="#42-用blackbox检测tcphttp服务状态">4.2 用blackbox检测TCP/HTTP服务状态</a>
          <ul>
            <li><a href="#421-被检测服务准备">4.2.1 被检测服务准备</a></li>
            <li><a href="#422-添加tcp的annotation">4.2.2 添加tcp的annotation</a></li>
            <li><a href="#423-添加http的annotation">4.2.3 添加http的annotation</a></li>
          </ul>
        </li>
        <li><a href="#43-添加监控jvm信息">4.3 添加监控jvm信息</a></li>
      </ul>
    </li>
  </ul>
</nav></div>
            </div><div class="content" id="content"><h1 id="14_k8s_监控实战-部署prometheus">14_K8S_监控实战-部署prometheus</h1>
<h2 id="1-prometheus前言相关">1 prometheus前言相关</h2>
<p>由于docker容器的特殊性，传统的zabbix无法对k8s集群内的docker状态进行监控，所以需要使用prometheus来进行监控
prometheus官网：<a href="https://prometheus.io/" target="_blank" rel="noopener noreffer">官网地址</a></p>
<h3 id="11-prometheus的特点">1.1 Prometheus的特点</h3>
<ul>
<li>多维度数据模型,使用时间序列数据库TSDB而不使用mysql。</li>
<li>灵活的查询语言PromQL。</li>
<li>不依赖分布式存储，单个服务器节点是自主的。</li>
<li>主要基于HTTP的pull方式主动采集时序数据</li>
<li>也可通过pushgateway获取主动推送到网关的数据。</li>
<li>通过服务发现或者静态配置来发现目标服务对象。</li>
<li>支持多种多样的图表和界面展示，比如Grafana等。</li>
</ul>
<h3 id="12-基本原理">1.2 基本原理</h3>
<h4 id="121-原理说明">1.2.1 原理说明</h4>
<p>Prometheus的基本原理是通过各种exporter提供的HTTP协议接口
周期性抓取被监控组件的状态，任意组件只要提供对应的HTTP接口就可以接入监控。
不需要任何SDK或者其他的集成过程,非常适合做虚拟化环境监控系统，比如VM、Docker、Kubernetes等。
互联网公司常用的组件大部分都有exporter可以直接使用，如Nginx、MySQL、Linux系统信息等。</p>
<h4 id="122-架构图">1.2.2 架构图:</h4>
<p><img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938866-fc30fe28-f729-49eb-a4b9-8cfeaa83ecb5.png"
        data-srcset="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938866-fc30fe28-f729-49eb-a4b9-8cfeaa83ecb5.png, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938866-fc30fe28-f729-49eb-a4b9-8cfeaa83ecb5.png 1.5x, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938866-fc30fe28-f729-49eb-a4b9-8cfeaa83ecb5.png 2x"
        data-sizes="auto"
        alt="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938866-fc30fe28-f729-49eb-a4b9-8cfeaa83ecb5.png"
        title="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938866-fc30fe28-f729-49eb-a4b9-8cfeaa83ecb5.png" /></p>
<h4 id="123-三大套件">1.2.3 三大套件</h4>
<ul>
<li>Server 主要负责数据采集和存储，提供PromQL查询语言的支持。</li>
<li>Alertmanager 警告管理器，用来进行报警。</li>
<li>Push Gateway 支持临时性Job主动推送指标的中间网关。</li>
</ul>
<h4 id="124-架构服务过程">1.2.4 架构服务过程</h4>
<ol>
<li>Prometheus Daemon负责定时去目标上抓取metrics(指标)数据
每个抓取目标需要暴露一个http服务的接口给它定时抓取。
支持通过配置文件、文本文件、Zookeeper、DNS SRV Lookup等方式指定抓取目标。</li>
<li>PushGateway用于Client主动推送metrics到PushGateway
而Prometheus只是定时去Gateway上抓取数据。
适合一次性、短生命周期的服务</li>
<li>Prometheus在TSDB数据库存储抓取的所有数据
通过一定规则进行清理和整理数据，并把得到的结果存储到新的时间序列中。</li>
<li>Prometheus通过PromQL和其他API可视化地展示收集的数据。
支持Grafana、Promdash等方式的图表数据可视化。
Prometheus还提供HTTP API的查询方式，自定义所需要的输出。</li>
<li>Alertmanager是独立于Prometheus的一个报警组件
支持Prometheus的查询语句，提供十分灵活的报警方式。</li>
</ol>
<h4 id="125-常用的exporter">1.2.5 常用的exporter</h4>
<p>prometheus不同于zabbix，没有agent，使用的是针对不同服务的exporter
正常情况下，监控k8s集群及node，pod，常用的exporter有四个：</p>
<ul>
<li>kube-state-metrics
收集k8s集群master&amp;etcd等基本状态信息</li>
<li>node-exporter
收集k8s集群node信息</li>
<li>cadvisor
收集k8s集群docker容器内部使用资源信息</li>
<li>blackbox-exporte
收集k8s集群docker容器服务是否存活</li>
</ul>
<h2 id="2-部署4个exporter">2 部署4个exporter</h2>
<p>老套路，下载docker镜像，准备资源配置清单，应用资源配置清单：</p>
<h3 id="21-部署kube-state-metrics">2.1 部署kube-state-metrics</h3>
<h4 id="211-准备docker镜像">2.1.1 准备docker镜像</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">docker pull quay.io/coreos/kube-state-metrics:v1.5.0
docker tag  91599517197a harbor.zq.com/public/kube-state-metrics:v1.5.0
docker push harbor.zq.com/public/kube-state-metrics:v1.5.0
</code></pre></td></tr></table>
</div>
</div><p>准备目录</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mkdir /data/k8s-yaml/kube-state-metrics
cd /data/k8s-yaml/kube-state-metrics
</code></pre></td></tr></table>
</div>
</div><h4 id="212-准备rbac资源清单">2.1.2 准备rbac资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span><span class="lnt">56
</span><span class="lnt">57
</span><span class="lnt">58
</span><span class="lnt">59
</span><span class="lnt">60
</span><span class="lnt">61
</span><span class="lnt">62
</span><span class="lnt">63
</span><span class="lnt">64
</span><span class="lnt">65
</span><span class="lnt">66
</span><span class="lnt">67
</span><span class="lnt">68
</span><span class="lnt">69
</span><span class="lnt">70
</span><span class="lnt">71
</span><span class="lnt">72
</span><span class="lnt">73
</span><span class="lnt">74
</span><span class="lnt">75
</span><span class="lnt">76
</span><span class="lnt">77
</span><span class="lnt">78
</span><span class="lnt">79
</span><span class="lnt">80
</span><span class="lnt">81
</span><span class="lnt">82
</span><span class="lnt">83
</span><span class="lnt">84
</span><span class="lnt">85
</span><span class="lnt">86
</span><span class="lnt">87
</span><span class="lnt">88
</span><span class="lnt">89
</span><span class="lnt">90
</span><span class="lnt">91
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;rbac.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/cluster-service: &#34;true&#34;
  name: kube-state-metrics
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/cluster-service: &#34;true&#34;
  name: kube-state-metrics
rules:
- apiGroups:
  - &#34;&#34;
  resources:
  - configmaps
  - secrets
  - nodes
  - pods
  - services
  - resourcequotas
  - replicationcontrollers
  - limitranges
  - persistentvolumeclaims
  - persistentvolumes
  - namespaces
  - endpoints
  verbs:
  - list
  - watch
- apiGroups:
  - policy
  resources:
  - poddisruptionbudgets
  verbs:
  - list
  - watch
- apiGroups:
  - extensions
  resources:
  - daemonsets
  - deployments
  - replicasets
  verbs:
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - statefulsets
  verbs:
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - cronjobs
  - jobs
  verbs:
  - list
  - watch
- apiGroups:
  - autoscaling
  resources:
  - horizontalpodautoscalers
  verbs:
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/cluster-service: &#34;true&#34;
  name: kube-state-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-state-metrics
subjects:
- kind: ServiceAccount
  name: kube-state-metrics
  namespace: kube-system
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="213-准备dp资源清单">2.1.3 准备Dp资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;dp.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: &#34;2&#34;
  labels:
    grafanak8sapp: &#34;true&#34;
    app: kube-state-metrics
  name: kube-state-metrics
  namespace: kube-system
spec:
  selector:
    matchLabels:
      grafanak8sapp: &#34;true&#34;
      app: kube-state-metrics
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        grafanak8sapp: &#34;true&#34;
        app: kube-state-metrics
    spec:
      containers:
      - name: kube-state-metrics
        image: harbor.zq.com/public/kube-state-metrics:v1.5.0
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
          name: http-metrics
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
      serviceAccountName: kube-state-metrics
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="214-应用资源配置清单">2.1.4 应用资源配置清单</h4>
<p>任意node节点执行</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl apply -f http://k8s-yaml.zq.com/kube-state-metrics/rbac.yaml
kubectl apply -f http://k8s-yaml.zq.com/kube-state-metrics/dp.yaml
</code></pre></td></tr></table>
</div>
</div><p><strong>验证测试</strong></p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl get pod -n kube-system -o wide|grep kube-state-metrices
~]# curl http://172.7.21.4:8080/healthz
ok
</code></pre></td></tr></table>
</div>
</div><p>返回OK表示已经成功运行。</p>
<h3 id="22-部署node-exporter">2.2 部署node-exporter</h3>
<p><strong>由于node-exporter是监控node的，需要每个节点启动一个，所以使用ds控制器</strong></p>
<h4 id="221-准备docker镜像">2.2.1 准备docker镜像</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">docker pull prom/node-exporter:v0.15.0
docker tag 12d51ffa2b22 harbor.zq.com/public/node-exporter:v0.15.0
docker push harbor.zq.com/public/node-exporter:v0.15.0
</code></pre></td></tr></table>
</div>
</div><p>准备目录</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mkdir /data/k8s-yaml/node-exporter
cd /data/k8s-yaml/node-exporter
</code></pre></td></tr></table>
</div>
</div><h4 id="222-准备ds资源清单">2.2.2 准备ds资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;ds.yaml &lt;&lt;&#39;EOF&#39;
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
  name: node-exporter
  namespace: kube-system
  labels:
    daemon: &#34;node-exporter&#34;
    grafanak8sapp: &#34;true&#34;
spec:
  selector:
    matchLabels:
      daemon: &#34;node-exporter&#34;
      grafanak8sapp: &#34;true&#34;
  template:
    metadata:
      name: node-exporter
      labels:
        daemon: &#34;node-exporter&#34;
        grafanak8sapp: &#34;true&#34;
    spec:
      volumes:
      - name: proc
        hostPath:
          path: /proc
          type: &#34;&#34;
      - name: sys
        hostPath:
          path: /sys
          type: &#34;&#34;
      containers:
      - name: node-exporter
        image: harbor.zq.com/public/node-exporter:v0.15.0
        imagePullPolicy: IfNotPresent
        args:
        - --path.procfs=/host_proc
        - --path.sysfs=/host_sys
        ports:
        - name: node-exporter
          hostPort: 9100
          containerPort: 9100
          protocol: TCP
        volumeMounts:
        - name: sys
          readOnly: true
          mountPath: /host_sys
        - name: proc
          readOnly: true
          mountPath: /host_proc
      hostNetwork: true
EOF
</code></pre></td></tr></table>
</div>
</div><blockquote>
<p>主要用途就是将宿主机的<code>/proc</code>,<code>sys</code>目录挂载给容器,是容器能获取node节点宿主机信息</p>
</blockquote>
<h4 id="223-应用资源配置清单">2.2.3 应用资源配置清单：</h4>
<p>任意node节点</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl apply -f http://k8s-yaml.zq.com/node-exporter/ds.yaml
kubectl get pod -n kube-system -o wide|grep node-exporter
</code></pre></td></tr></table>
</div>
</div><h3 id="23-部署cadvisor">2.3 部署cadvisor</h3>
<h4 id="231-准备docker镜像">2.3.1 准备docker镜像</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">docker pull google/cadvisor:v0.28.3
docker tag 75f88e3ec333 harbor.zq.com/public/cadvisor:0.28.3
docker push harbor.zq.com/public/cadvisor:0.28.3
</code></pre></td></tr></table>
</div>
</div><p>准备目录</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mkdir /data/k8s-yaml/cadvisor
cd /data/k8s-yaml/cadvisor
</code></pre></td></tr></table>
</div>
</div><h4 id="232-准备ds资源清单">2.3.2 准备ds资源清单</h4>
<p>cadvisor由于要获取每个node上的pod信息,因此也需要使用daemonset方式运行</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span><span class="lnt">56
</span><span class="lnt">57
</span><span class="lnt">58
</span><span class="lnt">59
</span><span class="lnt">60
</span><span class="lnt">61
</span><span class="lnt">62
</span><span class="lnt">63
</span><span class="lnt">64
</span><span class="lnt">65
</span><span class="lnt">66
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;ds.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: cadvisor
  namespace: kube-system
  labels:
    app: cadvisor
spec:
  selector:
    matchLabels:
      name: cadvisor
  template:
    metadata:
      labels:
        name: cadvisor
    spec:
      hostNetwork: true
#------pod的tolerations与node的Taints配合,做POD指定调度----
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
#-------------------------------------
      containers:
      - name: cadvisor
        image: harbor.zq.com/public/cadvisor:v0.28.3
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: rootfs
          mountPath: /rootfs
          readOnly: true
        - name: var-run
          mountPath: /var/run
        - name: sys
          mountPath: /sys
          readOnly: true
        - name: docker
          mountPath: /var/lib/docker
          readOnly: true
        ports:
          - name: http
            containerPort: 4194
            protocol: TCP
        readinessProbe:
          tcpSocket:
            port: 4194
          initialDelaySeconds: 5
          periodSeconds: 10
        args:
          - --housekeeping_interval=10s
          - --port=4194
      terminationGracePeriodSeconds: 30
      volumes:
      - name: rootfs
        hostPath:
          path: /
      - name: var-run
        hostPath:
          path: /var/run
      - name: sys
        hostPath:
          path: /sys
      - name: docker
        hostPath:
          path: /data/docker
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="233-应用资源配置清单">2.3.3 应用资源配置清单：</h4>
<p>应用清单前,先在每个node上做以下软连接,否则服务可能报错</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mount -o remount,rw /sys/fs/cgroup/
ln -s /sys/fs/cgroup/cpu,cpuacct /sys/fs/cgroup/cpuacct,cpu
</code></pre></td></tr></table>
</div>
</div><p>应用清单</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl apply -f http://k8s-yaml.zq.com/cadvisor/ds.yaml
</code></pre></td></tr></table>
</div>
</div><p>检查：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl -n kube-system get pod -o wide|grep cadvisor
</code></pre></td></tr></table>
</div>
</div><h3 id="24-部署blackbox-exporter">2.4 部署blackbox-exporter</h3>
<h4 id="241-准备docker镜像">2.4.1 准备docker镜像</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">docker pull prom/blackbox-exporter:v0.15.1
docker tag  81b70b6158be  harbor.zq.com/public/blackbox-exporter:v0.15.1
docker push harbor.zq.com/public/blackbox-exporter:v0.15.1
</code></pre></td></tr></table>
</div>
</div><p>准备目录</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mkdir /data/k8s-yaml/blackbox-exporter
cd /data/k8s-yaml/blackbox-exporter
</code></pre></td></tr></table>
</div>
</div><h4 id="242-准备cm资源清单">2.4.2 准备cm资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;cm.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    app: blackbox-exporter
  name: blackbox-exporter
  namespace: kube-system
data:
  blackbox.yml: |-
    modules:
      http_2xx:
        prober: http
        timeout: 2s
        http:
          valid_http_versions: [&#34;HTTP/1.1&#34;, &#34;HTTP/2&#34;]
          valid_status_codes: [200,301,302]
          method: GET
          preferred_ip_protocol: &#34;ip4&#34;
      tcp_connect:
        prober: tcp
        timeout: 2s
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="243-准备dp资源清单">2.4.3 准备dp资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span><span class="lnt">56
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;dp.yaml &lt;&lt;&#39;EOF&#39;
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: blackbox-exporter
  namespace: kube-system
  labels:
    app: blackbox-exporter
  annotations:
    deployment.kubernetes.io/revision: 1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: blackbox-exporter
  template:
    metadata:
      labels:
        app: blackbox-exporter
    spec:
      volumes:
      - name: config
        configMap:
          name: blackbox-exporter
          defaultMode: 420
      containers:
      - name: blackbox-exporter
        image: harbor.zq.com/public/blackbox-exporter:v0.15.1
        imagePullPolicy: IfNotPresent
        args:
        - --config.file=/etc/blackbox_exporter/blackbox.yml
        - --log.level=info
        - --web.listen-address=:9115
        ports:
        - name: blackbox-port
          containerPort: 9115
          protocol: TCP
        resources:
          limits:
            cpu: 200m
            memory: 256Mi
          requests:
            cpu: 100m
            memory: 50Mi
        volumeMounts:
        - name: config
          mountPath: /etc/blackbox_exporter
        readinessProbe:
          tcpSocket:
            port: 9115
          initialDelaySeconds: 5
          timeoutSeconds: 5
          periodSeconds: 10
          successThreshold: 1
          failureThreshold: 3
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="244-准备svc资源清单">2.4.4 准备svc资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;svc.yaml &lt;&lt;&#39;EOF&#39;
kind: Service
apiVersion: v1
metadata:
  name: blackbox-exporter
  namespace: kube-system
spec:
  selector:
    app: blackbox-exporter
  ports:
    - name: blackbox-port
      protocol: TCP
      port: 9115
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="245-准备ingress资源清单">2.4.5 准备ingress资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;ingress.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: blackbox-exporter
  namespace: kube-system
spec:
  rules:
  - host: blackbox.zq.com
    http:
      paths:
      - path: /
        backend:
          serviceName: blackbox-exporter
          servicePort: blackbox-port
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="246-添加域名解析">2.4.6 添加域名解析</h4>
<p>这里用到了一个域名，添加解析</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">vi /var/named/zq.com.zone
blackbox       A    10.4.7.10
systemctl restart named
</code></pre></td></tr></table>
</div>
</div><h4 id="247-应用资源配置清单">2.4.7 应用资源配置清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl apply -f http://k8s-yaml.zq.com/blackbox-exporter/cm.yaml
kubectl apply -f http://k8s-yaml.zq.com/blackbox-exporter/dp.yaml
kubectl apply -f http://k8s-yaml.zq.com/blackbox-exporter/svc.yaml
kubectl apply -f http://k8s-yaml.zq.com/blackbox-exporter/ingress.yaml
</code></pre></td></tr></table>
</div>
</div><h4 id="248-访问域名测试">2.4.8 访问域名测试</h4>
<p>访问<a href="http://blackbox.zq.com" target="_blank" rel="noopener noreffer">http://blackbox.zq.com</a>，显示如下界面,表示blackbox已经运行成
<img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938890-ac98286e-bfe4-40be-91ad-b1d024a5b92b.png"
        data-srcset="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938890-ac98286e-bfe4-40be-91ad-b1d024a5b92b.png, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938890-ac98286e-bfe4-40be-91ad-b1d024a5b92b.png 1.5x, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938890-ac98286e-bfe4-40be-91ad-b1d024a5b92b.png 2x"
        data-sizes="auto"
        alt="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938890-ac98286e-bfe4-40be-91ad-b1d024a5b92b.png"
        title="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938890-ac98286e-bfe4-40be-91ad-b1d024a5b92b.png" /></p>
<h2 id="3-部署prometheus-server">3 部署prometheus server</h2>
<h3 id="31-准备prometheus-server环境">3.1 准备prometheus server环境</h3>
<h4 id="311-准备docker镜像">3.1.1 准备docker镜像</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">docker pull prom/prometheus:v2.14.0
docker tag  7317640d555e harbor.zq.com/infra/prometheus:v2.14.0
docker push harbor.zq.com/infra/prometheus:v2.14.0
</code></pre></td></tr></table>
</div>
</div><p>准备目录</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mkdir /data/k8s-yaml/prometheus-server
cd /data/k8s-yaml/prometheus-server
</code></pre></td></tr></table>
</div>
</div><h4 id="312-准备rbac资源清单">3.1.2 准备rbac资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span><span class="lnt">56
</span><span class="lnt">57
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;rbac.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/cluster-service: &#34;true&#34;
  name: prometheus
  namespace: infra
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/cluster-service: &#34;true&#34;
  name: prometheus
rules:
- apiGroups:
  - &#34;&#34;
  resources:
  - nodes
  - nodes/metrics
  - services
  - endpoints
  - pods
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - &#34;&#34;
  resources:
  - configmaps
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/cluster-service: &#34;true&#34;
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: infra
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="313-准备dp资源清单">3.1.3 准备dp资源清单</h4>
<blockquote>
<p>加上<code>--web.enable-lifecycle</code>启用远程热加载配置文件,配置文件改变后不用重启prometheus
调用指令是<code>curl -X POST http://localhost:9090/-/reload</code>
<code>storage.tsdb.min-block-duration=10m</code>只加载10分钟数据到内
<code>storage.tsdb.retention=72h</code> 保留72小时数据</p>
</blockquote>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span><span class="lnt">56
</span><span class="lnt">57
</span><span class="lnt">58
</span><span class="lnt">59
</span><span class="lnt">60
</span><span class="lnt">61
</span><span class="lnt">62
</span><span class="lnt">63
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;dp.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: &#34;5&#34;
  labels:
    name: prometheus
  name: prometheus
  namespace: infra
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 7
  selector:
    matchLabels:
      app: prometheus
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      containers:
      - name: prometheus
        image: harbor.zq.com/infra/prometheus:v2.14.0
        imagePullPolicy: IfNotPresent
        command:
        - /bin/prometheus
        args:
        - --config.file=/data/etc/prometheus.yml
        - --storage.tsdb.path=/data/prom-db
        - --storage.tsdb.min-block-duration=10m
        - --storage.tsdb.retention=72h
        - --web.enable-lifecycle
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - mountPath: /data
          name: data
        resources:
          requests:
            cpu: &#34;1000m&#34;
            memory: &#34;1.5Gi&#34;
          limits:
            cpu: &#34;2000m&#34;
            memory: &#34;3Gi&#34;
      imagePullSecrets:
      - name: harbor
      securityContext:
        runAsUser: 0
      serviceAccountName: prometheus
      volumes:
      - name: data
        nfs:
          server: hdss7-200
          path: /data/nfs-volume/prometheus
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="314-准备svc资源清单">3.1.4 准备svc资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;svc.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: infra
spec:
  ports:
  - port: 9090
    protocol: TCP
    targetPort: 9090
  selector:
    app: prometheus
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="315-准备ingress资源清单">3.1.5 准备ingress资源清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;ingress.yaml &lt;&lt;&#39;EOF&#39;
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: traefik
  name: prometheus
  namespace: infra
spec:
  rules:
  - host: prometheus.zq.com
    http:
      paths:
      - path: /
        backend:
          serviceName: prometheus
          servicePort: 9090
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="316-添加域名解析">3.1.6 添加域名解析</h4>
<p>这里用到一个域名<code>prometheus.zq.com</code>，添加解析：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">vi /var/named/od.com.zone
prometheus         A    10.4.7.10
systemctl restart named
</code></pre></td></tr></table>
</div>
</div><h3 id="32-部署prometheus-server">3.2 部署prometheus server</h3>
<h4 id="321-准备目录和证书">3.2.1 准备目录和证书</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mkdir -p /data/nfs-volume/prometheus/etc
mkdir -p /data/nfs-volume/prometheus/prom-db
cd /data/nfs-volume/prometheus/etc/
# 拷贝配置文件中用到的证书：
cp /opt/certs/ca.pem ./
cp /opt/certs/client.pem ./
cp /opt/certs/client-key.pem ./
</code></pre></td></tr></table>
</div>
</div><h4 id="322-创建prometheus配置文件">3.2.2 创建prometheus配置文件</h4>
<blockquote>
<p><strong>配置文件说明:</strong>
此配置为通用配置,除第一个job<code>etcd</code>是做的静态配置外,其他8个job都是做的自动发现
因此只需要修改<code>etcd</code>的配置后,就可以直接用于生产环境</p>
</blockquote>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">  1
</span><span class="lnt">  2
</span><span class="lnt">  3
</span><span class="lnt">  4
</span><span class="lnt">  5
</span><span class="lnt">  6
</span><span class="lnt">  7
</span><span class="lnt">  8
</span><span class="lnt">  9
</span><span class="lnt"> 10
</span><span class="lnt"> 11
</span><span class="lnt"> 12
</span><span class="lnt"> 13
</span><span class="lnt"> 14
</span><span class="lnt"> 15
</span><span class="lnt"> 16
</span><span class="lnt"> 17
</span><span class="lnt"> 18
</span><span class="lnt"> 19
</span><span class="lnt"> 20
</span><span class="lnt"> 21
</span><span class="lnt"> 22
</span><span class="lnt"> 23
</span><span class="lnt"> 24
</span><span class="lnt"> 25
</span><span class="lnt"> 26
</span><span class="lnt"> 27
</span><span class="lnt"> 28
</span><span class="lnt"> 29
</span><span class="lnt"> 30
</span><span class="lnt"> 31
</span><span class="lnt"> 32
</span><span class="lnt"> 33
</span><span class="lnt"> 34
</span><span class="lnt"> 35
</span><span class="lnt"> 36
</span><span class="lnt"> 37
</span><span class="lnt"> 38
</span><span class="lnt"> 39
</span><span class="lnt"> 40
</span><span class="lnt"> 41
</span><span class="lnt"> 42
</span><span class="lnt"> 43
</span><span class="lnt"> 44
</span><span class="lnt"> 45
</span><span class="lnt"> 46
</span><span class="lnt"> 47
</span><span class="lnt"> 48
</span><span class="lnt"> 49
</span><span class="lnt"> 50
</span><span class="lnt"> 51
</span><span class="lnt"> 52
</span><span class="lnt"> 53
</span><span class="lnt"> 54
</span><span class="lnt"> 55
</span><span class="lnt"> 56
</span><span class="lnt"> 57
</span><span class="lnt"> 58
</span><span class="lnt"> 59
</span><span class="lnt"> 60
</span><span class="lnt"> 61
</span><span class="lnt"> 62
</span><span class="lnt"> 63
</span><span class="lnt"> 64
</span><span class="lnt"> 65
</span><span class="lnt"> 66
</span><span class="lnt"> 67
</span><span class="lnt"> 68
</span><span class="lnt"> 69
</span><span class="lnt"> 70
</span><span class="lnt"> 71
</span><span class="lnt"> 72
</span><span class="lnt"> 73
</span><span class="lnt"> 74
</span><span class="lnt"> 75
</span><span class="lnt"> 76
</span><span class="lnt"> 77
</span><span class="lnt"> 78
</span><span class="lnt"> 79
</span><span class="lnt"> 80
</span><span class="lnt"> 81
</span><span class="lnt"> 82
</span><span class="lnt"> 83
</span><span class="lnt"> 84
</span><span class="lnt"> 85
</span><span class="lnt"> 86
</span><span class="lnt"> 87
</span><span class="lnt"> 88
</span><span class="lnt"> 89
</span><span class="lnt"> 90
</span><span class="lnt"> 91
</span><span class="lnt"> 92
</span><span class="lnt"> 93
</span><span class="lnt"> 94
</span><span class="lnt"> 95
</span><span class="lnt"> 96
</span><span class="lnt"> 97
</span><span class="lnt"> 98
</span><span class="lnt"> 99
</span><span class="lnt">100
</span><span class="lnt">101
</span><span class="lnt">102
</span><span class="lnt">103
</span><span class="lnt">104
</span><span class="lnt">105
</span><span class="lnt">106
</span><span class="lnt">107
</span><span class="lnt">108
</span><span class="lnt">109
</span><span class="lnt">110
</span><span class="lnt">111
</span><span class="lnt">112
</span><span class="lnt">113
</span><span class="lnt">114
</span><span class="lnt">115
</span><span class="lnt">116
</span><span class="lnt">117
</span><span class="lnt">118
</span><span class="lnt">119
</span><span class="lnt">120
</span><span class="lnt">121
</span><span class="lnt">122
</span><span class="lnt">123
</span><span class="lnt">124
</span><span class="lnt">125
</span><span class="lnt">126
</span><span class="lnt">127
</span><span class="lnt">128
</span><span class="lnt">129
</span><span class="lnt">130
</span><span class="lnt">131
</span><span class="lnt">132
</span><span class="lnt">133
</span><span class="lnt">134
</span><span class="lnt">135
</span><span class="lnt">136
</span><span class="lnt">137
</span><span class="lnt">138
</span><span class="lnt">139
</span><span class="lnt">140
</span><span class="lnt">141
</span><span class="lnt">142
</span><span class="lnt">143
</span><span class="lnt">144
</span><span class="lnt">145
</span><span class="lnt">146
</span><span class="lnt">147
</span><span class="lnt">148
</span><span class="lnt">149
</span><span class="lnt">150
</span><span class="lnt">151
</span><span class="lnt">152
</span><span class="lnt">153
</span><span class="lnt">154
</span><span class="lnt">155
</span><span class="lnt">156
</span><span class="lnt">157
</span><span class="lnt">158
</span><span class="lnt">159
</span><span class="lnt">160
</span><span class="lnt">161
</span><span class="lnt">162
</span><span class="lnt">163
</span><span class="lnt">164
</span><span class="lnt">165
</span><span class="lnt">166
</span><span class="lnt">167
</span><span class="lnt">168
</span><span class="lnt">169
</span><span class="lnt">170
</span><span class="lnt">171
</span><span class="lnt">172
</span><span class="lnt">173
</span><span class="lnt">174
</span><span class="lnt">175
</span><span class="lnt">176
</span><span class="lnt">177
</span><span class="lnt">178
</span><span class="lnt">179
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;/data/nfs-volume/prometheus/etc/prometheus.yml &lt;&lt;&#39;EOF&#39;
global:
  scrape_interval:     15s
  evaluation_interval: 15s
scrape_configs:
- job_name: &#39;etcd&#39;
  tls_config:
    ca_file: /data/etc/ca.pem
    cert_file: /data/etc/client.pem
    key_file: /data/etc/client-key.pem
  scheme: https
  static_configs:
  - targets:
    - &#39;10.4.7.12:2379&#39;
    - &#39;10.4.7.21:2379&#39;
    - &#39;10.4.7.22:2379&#39;

- job_name: &#39;kubernetes-apiservers&#39;
  kubernetes_sd_configs:
  - role: endpoints
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  relabel_configs:
  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
    action: keep
    regex: default;kubernetes;https

- job_name: &#39;kubernetes-pods&#39;
  kubernetes_sd_configs:
  - role: pod
  relabel_configs:
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
    action: replace
    target_label: __metrics_path__
    regex: (.+)
  - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
    action: replace
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
    target_label: __address__
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name

- job_name: &#39;kubernetes-kubelet&#39;
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - source_labels: [__meta_kubernetes_node_name]
    regex: (.+)
    target_label: __address__
    replacement: ${1}:10255

- job_name: &#39;kubernetes-cadvisor&#39;
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - source_labels: [__meta_kubernetes_node_name]
    regex: (.+)
    target_label: __address__
    replacement: ${1}:4194

- job_name: &#39;kubernetes-kube-state&#39;
  kubernetes_sd_configs:
  - role: pod
  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name
  - source_labels: [__meta_kubernetes_pod_label_grafanak8sapp]
    regex: .*true.*
    action: keep
  - source_labels: [&#39;__meta_kubernetes_pod_label_daemon&#39;, &#39;__meta_kubernetes_pod_node_name&#39;]
    regex: &#39;node-exporter;(.*)&#39;
    action: replace
    target_label: nodename

- job_name: &#39;blackbox_http_pod_probe&#39;
  metrics_path: /probe
  kubernetes_sd_configs:
  - role: pod
  params:
    module: [http_2xx]
  relabel_configs:
  - source_labels: [__meta_kubernetes_pod_annotation_blackbox_scheme]
    action: keep
    regex: http
  - source_labels: [__address__, __meta_kubernetes_pod_annotation_blackbox_port,  __meta_kubernetes_pod_annotation_blackbox_path]
    action: replace
    regex: ([^:]+)(?::\d+)?;(\d+);(.+)
    replacement: $1:$2$3
    target_label: __param_target
  - action: replace
    target_label: __address__
    replacement: blackbox-exporter.kube-system:9115
  - source_labels: [__param_target]
    target_label: instance
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name

- job_name: &#39;blackbox_tcp_pod_probe&#39;
  metrics_path: /probe
  kubernetes_sd_configs:
  - role: pod
  params:
    module: [tcp_connect]
  relabel_configs:
  - source_labels: [__meta_kubernetes_pod_annotation_blackbox_scheme]
    action: keep
    regex: tcp
  - source_labels: [__address__, __meta_kubernetes_pod_annotation_blackbox_port]
    action: replace
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
    target_label: __param_target
  - action: replace
    target_label: __address__
    replacement: blackbox-exporter.kube-system:9115
  - source_labels: [__param_target]
    target_label: instance
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name

- job_name: &#39;traefik&#39;
  kubernetes_sd_configs:
  - role: pod
  relabel_configs:
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
    action: keep
    regex: traefik
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
    action: replace
    target_label: __metrics_path__
    regex: (.+)
  - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
    action: replace
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
    target_label: __address__
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name
EOF
</code></pre></td></tr></table>
</div>
</div><h4 id="323-应用资源配置清单">3.2.3 应用资源配置清单</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl apply -f http://k8s-yaml.zq.com/prometheus-server/rbac.yaml
kubectl apply -f http://k8s-yaml.zq.com/prometheus-server/dp.yaml
kubectl apply -f http://k8s-yaml.zq.com/prometheus-server/svc.yaml
kubectl apply -f http://k8s-yaml.zq.com/prometheus-server/ingress.yaml
</code></pre></td></tr></table>
</div>
</div><h4 id="324-浏览器验证">3.2.4 浏览器验证</h4>
<p>访问<a href="http://prometheus.zq.com," target="_blank" rel="noopener noreffer">http://prometheus.zq.com,</a>如果能成功访问的话,表示启动成功
点击<strong>status-&gt;configuration</strong>就是我们的配置文件
<img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://cdn.nlark.com/yuque/0/2020/png/2511954/1601219938865-a380d765-1bde-4948-9878-46528803c465.png#align=left&amp;display=inline&amp;height=333&amp;margin=%5Bobject%20Object%5D&amp;originHeight=333&amp;originWidth=496&amp;size=0&amp;status=done&amp;style=none&amp;width=496"
        data-srcset="https://cdn.nlark.com/yuque/0/2020/png/2511954/1601219938865-a380d765-1bde-4948-9878-46528803c465.png#align=left&amp;display=inline&amp;height=333&amp;margin=%5Bobject%20Object%5D&amp;originHeight=333&amp;originWidth=496&amp;size=0&amp;status=done&amp;style=none&amp;width=496, https://cdn.nlark.com/yuque/0/2020/png/2511954/1601219938865-a380d765-1bde-4948-9878-46528803c465.png#align=left&amp;display=inline&amp;height=333&amp;margin=%5Bobject%20Object%5D&amp;originHeight=333&amp;originWidth=496&amp;size=0&amp;status=done&amp;style=none&amp;width=496 1.5x, https://cdn.nlark.com/yuque/0/2020/png/2511954/1601219938865-a380d765-1bde-4948-9878-46528803c465.png#align=left&amp;display=inline&amp;height=333&amp;margin=%5Bobject%20Object%5D&amp;originHeight=333&amp;originWidth=496&amp;size=0&amp;status=done&amp;style=none&amp;width=496 2x"
        data-sizes="auto"
        alt="https://cdn.nlark.com/yuque/0/2020/png/2511954/1601219938865-a380d765-1bde-4948-9878-46528803c465.png#align=left&amp;display=inline&amp;height=333&amp;margin=%5Bobject%20Object%5D&amp;originHeight=333&amp;originWidth=496&amp;size=0&amp;status=done&amp;style=none&amp;width=496"
        title="https://cdn.nlark.com/yuque/0/2020/png/2511954/1601219938865-a380d765-1bde-4948-9878-46528803c465.png#align=left&amp;display=inline&amp;height=333&amp;margin=%5Bobject%20Object%5D&amp;originHeight=333&amp;originWidth=496&amp;size=0&amp;status=done&amp;style=none&amp;width=496" /></p>
<h2 id="4-使服务能被prometheus自动监控">4 使服务能被prometheus自动监控</h2>
<p>点击<strong>status-&gt;targets</strong>，展示的就是我们在prometheus.yml中配置的job-name，这些targets基本可以满足我们收集数据的需求。
<img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-db8767e1-361c-415d-bb55-b6f7bf37db50.png"
        data-srcset="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-db8767e1-361c-415d-bb55-b6f7bf37db50.png, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-db8767e1-361c-415d-bb55-b6f7bf37db50.png 1.5x, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-db8767e1-361c-415d-bb55-b6f7bf37db50.png 2x"
        data-sizes="auto"
        alt="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-db8767e1-361c-415d-bb55-b6f7bf37db50.png"
        title="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-db8767e1-361c-415d-bb55-b6f7bf37db50.png" /></p>
<blockquote>
<p>5个编号的job-name已经被发现并获取数据
接下来就需要将剩下的4个ob-name对应的服务纳入监控
纳入监控的方式是给需要收集数据的服务添加annotations</p>
</blockquote>
<h3 id="41-让traefik能被自动监控">4.1 让traefik能被自动监控</h3>
<h4 id="411-修改traefik的yaml">4.1.1 修改traefik的yaml</h4>
<p>修改fraefik的yaml文件,跟labels同级,添加annotations配置</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">vim /data/k8s-yaml/traefik/ds.yaml
........
spec:
  template:
    metadata:
      labels:
        k8s-app: traefik-ingress
        name: traefik-ingress
#--------增加内容--------
      annotations:
        prometheus_io_scheme: &#34;traefik&#34;
        prometheus_io_path: &#34;/metrics&#34;
        prometheus_io_port: &#34;8080&#34;
#--------增加结束--------
    spec:
      serviceAccountName: traefik-ingress-controller
........
</code></pre></td></tr></table>
</div>
</div><p>任意节点重新应用配置</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl delete -f http://k8s-yaml.zq.com/traefik/ds.yaml
kubectl apply  -f http://k8s-yaml.zq.com/traefik/ds.yaml
</code></pre></td></tr></table>
</div>
</div><h4 id="412-应用配置查看">4.1.2 应用配置查看</h4>
<p>等待pod重启以后，再在prometheus上查看traefik是否能正常获取数据了
<img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938921-6fc54a66-c82d-4886-bcba-839ad5a311b2.png"
        data-srcset="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938921-6fc54a66-c82d-4886-bcba-839ad5a311b2.png, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938921-6fc54a66-c82d-4886-bcba-839ad5a311b2.png 1.5x, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938921-6fc54a66-c82d-4886-bcba-839ad5a311b2.png 2x"
        data-sizes="auto"
        alt="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938921-6fc54a66-c82d-4886-bcba-839ad5a311b2.png"
        title="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938921-6fc54a66-c82d-4886-bcba-839ad5a311b2.png" /></p>
<h3 id="42-用blackbox检测tcphttp服务状态">4.2 用blackbox检测TCP/HTTP服务状态</h3>
<p>blackbox是检测容器内服务存活性的，也就是端口健康状态检查，分为tcp和http两种方法
能用http的情况尽量用http,没有提供http接口的服务才用tcp</p>
<h4 id="421-被检测服务准备">4.2.1 被检测服务准备</h4>
<p>使用测试环境的dubbo服务来做演示,其他环境类似</p>
<ol>
<li>dashboard中开启apollo-portal和test空间中的apollo</li>
<li>dubbo-demo-service使用tcp的annotation</li>
<li>dubbo-demo-consumer使用HTTP的annotation</li>
</ol>
<h4 id="422-添加tcp的annotation">4.2.2 添加tcp的annotation</h4>
<p>等两个服务起来以后，首先在dubbo-demo-service资源中添加一个TCP的annotation</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">vim /data/k8s-yaml/test/dubbo-demo-server/dp.yaml
......
spec:
......
  template:
    metadata:
      labels:
        app: dubbo-demo-service
        name: dubbo-demo-service
#--------增加内容--------
      annotations:
        blackbox_port: &#34;20880&#34;
        blackbox_scheme: &#34;tcp&#34;
#--------增加结束--------
    spec:
      containers:
        image: harbor.zq.com/app/dubbo-demo-service:apollo_200512_0746
</code></pre></td></tr></table>
</div>
</div><p>任意节点重新应用配置</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl delete -f http://k8s-yaml.zq.com/test/dubbo-demo-server/dp.yaml
kubectl apply  -f http://k8s-yaml.zq.com/test/dubbo-demo-server/dp.yaml
</code></pre></td></tr></table>
</div>
</div><p>浏览器中查看<a href="http://blackbox.zq.com/" target="_blank" rel="noopener noreffer">http://blackbox.zq.com/</a>和<a href="http://prometheus.zq.com/targets" target="_blank" rel="noopener noreffer">http://prometheus.zq.com/targets</a>
我们运行的dubbo-demo-server服务,tcp端口20880已经被发现并在监控中
<img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-2fc3c386-8185-4e5a-9692-86524c14201e.png"
        data-srcset="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-2fc3c386-8185-4e5a-9692-86524c14201e.png, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-2fc3c386-8185-4e5a-9692-86524c14201e.png 1.5x, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-2fc3c386-8185-4e5a-9692-86524c14201e.png 2x"
        data-sizes="auto"
        alt="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-2fc3c386-8185-4e5a-9692-86524c14201e.png"
        title="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938915-2fc3c386-8185-4e5a-9692-86524c14201e.png" /></p>
<h4 id="423-添加http的annotation">4.2.3 添加http的annotation</h4>
<p>接下来在dubbo-demo-consumer资源中添加一个HTTP的annotation：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">vim /data/k8s-yaml/test/dubbo-demo-consumer/dp.yaml
spec:
......
  template:
    metadata:
      labels:
        app: dubbo-demo-consumer
        name: dubbo-demo-consumer
#--------增加内容--------
      annotations:
        blackbox_path: &#34;/hello?name=health&#34;
        blackbox_port: &#34;8080&#34;
        blackbox_scheme: &#34;http&#34;
#--------增加结束--------
    spec:
      containers:
      - name: dubbo-demo-consumer
......
</code></pre></td></tr></table>
</div>
</div><p>任意节点重新应用配置</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl delete -f http://k8s-yaml.zq.com/test/dubbo-demo-consumer/dp.yaml
kubectl apply  -f http://k8s-yaml.zq.com/test/dubbo-demo-consumer/dp.yaml
</code></pre></td></tr></table>
</div>
</div><p><img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938954-b889a59e-91de-4c2b-85b7-36f390c529ad.png"
        data-srcset="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938954-b889a59e-91de-4c2b-85b7-36f390c529ad.png, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938954-b889a59e-91de-4c2b-85b7-36f390c529ad.png 1.5x, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938954-b889a59e-91de-4c2b-85b7-36f390c529ad.png 2x"
        data-sizes="auto"
        alt="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938954-b889a59e-91de-4c2b-85b7-36f390c529ad.png"
        title="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938954-b889a59e-91de-4c2b-85b7-36f390c529ad.png" /></p>
<h3 id="43-添加监控jvm信息">4.3 添加监控jvm信息</h3>
<p>dubbo-demo-service和dubbo-demo-consumer都添加下列annotation注解,以便监控pod中的jvm信息</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">vim /data/k8s-yaml/test/dubbo-demo-server/dp.yaml
vim /data/k8s-yaml/test/dubbo-demo-consumer/dp.yaml
      annotations:
        #....已有略....
        prometheus_io_scrape: &#34;true&#34;
        prometheus_io_port: &#34;12346&#34;
        prometheus_io_path: &#34;/&#34;
</code></pre></td></tr></table>
</div>
</div><blockquote>
<p>12346是dubbo的POD启动命令中使用jmx_javaagent用到的端口,因此可以用来收集jvm信息</p>
</blockquote>
<p>任意节点重新应用配置</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">kubectl apply  -f http://k8s-yaml.zq.com/test/dubbo-demo-server/dp.yaml
kubectl apply  -f http://k8s-yaml.zq.com/test/dubbo-demo-consumer/dp.yaml
</code></pre></td></tr></table>
</div>
</div><p><img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938896-309a3f84-71c6-4d02-8245-4d782ee272af.png"
        data-srcset="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938896-309a3f84-71c6-4d02-8245-4d782ee272af.png, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938896-309a3f84-71c6-4d02-8245-4d782ee272af.png 1.5x, https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938896-309a3f84-71c6-4d02-8245-4d782ee272af.png 2x"
        data-sizes="auto"
        alt="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938896-309a3f84-71c6-4d02-8245-4d782ee272af.png"
        title="https://jyd01.oss-cn-beijing.aliyuncs.com/uPic/1601219938896-309a3f84-71c6-4d02-8245-4d782ee272af.png" /></p>
<p><strong>至此,所有9个服务,都获取了数据</strong></p>
</div><div class="post-footer" id="post-footer">
    <div class="post-info">
        <div class="post-info-line">
            <div class="post-info-mod">
                <span>更新于 2020-10-01</span>
            </div>
            <div class="post-info-license"></div>
        </div>
        <div class="post-info-line">
            <div class="post-info-md"></div>
            <div class="post-info-share">
                <span><a href="javascript:void(0);" title="分享到 Twitter" data-sharer="twitter" data-url="https://scemsjyd.com/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus/" data-title="14_K8S_监控实战-部署prometheus" data-hashtags="K8S,转载"><i class="fab fa-twitter fa-fw"></i></a><a href="javascript:void(0);" title="分享到 WhatsApp" data-sharer="whatsapp" data-url="https://scemsjyd.com/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus/" data-title="14_K8S_监控实战-部署prometheus" data-web><i class="fab fa-whatsapp fa-fw"></i></a><a href="javascript:void(0);" title="分享到 微博" data-sharer="weibo" data-url="https://scemsjyd.com/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus/" data-title="14_K8S_监控实战-部署prometheus"><i class="fab fa-weibo fa-fw"></i></a><a href="javascript:void(0);" title="分享到 Evernote" data-sharer="evernote" data-url="https://scemsjyd.com/14_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-%E9%83%A8%E7%BD%B2prometheus/" data-title="14_K8S_监控实战-部署prometheus"><i class="fab fa-evernote fa-fw"></i></a></span>
            </div>
        </div>
    </div>

    <div class="post-info-more">
        <section class="post-tags"><i class="fas fa-tags fa-fw"></i>&nbsp;<a href="/tags/k8s/">K8S</a>,&nbsp;<a href="/tags/%E8%BD%AC%E8%BD%BD/">转载</a></section>
        <section>
            <span><a href="javascript:void(0);" onclick="window.history.back();">返回</a></span>&nbsp;|&nbsp;<span><a href="/">主页</a></span>
        </section>
    </div>

    <div class="post-nav"><a href="/15_k8s_%E7%9B%91%E6%8E%A7%E5%AE%9E%E6%88%98-grafana%E5%87%BA%E5%9B%BE_alert%E5%91%8A%E8%AD%A6/" class="prev" rel="prev" title="15_K8S_监控实战-grafana出图_alert告警"><i class="fas fa-angle-left fa-fw"></i>15_K8S_监控实战-grafana出图_alert告警</a>
            <a href="/13_k8s_%E9%85%8D%E7%BD%AE%E4%B8%AD%E5%BF%83%E5%AE%9E%E6%88%98-%E5%A4%9A%E7%8E%AF%E5%A2%83%E4%BA%A4%E4%BB%98apollo%E4%B8%89%E7%BB%84%E4%BB%B6/" class="next" rel="next" title="13_K8S_配置中心实战-多环境交付apollo三组件">13_K8S_配置中心实战-多环境交付apollo三组件<i class="fas fa-angle-right fa-fw"></i></a></div>
</div>
<div id="comments"><div id="valine" class="comment"></div><noscript>
                Please enable JavaScript to view the comments powered by <a href="https://valine.js.org/">Valine</a>.
            </noscript></div></article></div>
            </main><footer class="footer">
        <div class="footer-container"><div class="footer-line"><span><a href='https://www.upyun.com/?utm_source=lianmeng&utm_medium=referral'>本网站由<img style='vertical-align:bottom' width='60' src='/images/upyun.svg'>提供CDN加速/云存储服务</a></span></div><div class="footer-line"><i class="far fa-copyright fa-fw"></i><span itemprop="copyrightYear">2016 - 2022</span><span class="author" itemprop="copyrightHolder">&nbsp;<a href="/" target="_blank">Adam.Jin</a></span>&nbsp;|&nbsp;<span class="license"><a rel="license external nofollow noopener noreffer" href="https://creativecommons.org/licenses/by-nc/4.0/" target="_blank">CC BY-NC 4.0</a></span><span class="icp-splitter">&nbsp;|&nbsp;</span><br class="icp-br"/>
                    <span class="icp"><a href='https://beian.miit.gov.cn/'>蜀ICP备2021027971号-1</a></span></div>
        </div>
    </footer></div>

        <div id="fixed-buttons"><a href="#" id="back-to-top" class="fixed-button" title="回到顶部">
                <i class="fas fa-arrow-up fa-fw"></i>
            </a><a href="#" id="view-comments" class="fixed-button" title="查看评论">
                <i class="fas fa-comment fa-fw"></i>
            </a>
        </div><link rel="stylesheet" href="/lib/valine/valine.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/lightgallery.js@1.2.0/dist/css/lightgallery.min.css"><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/valine@1.4.14/dist/Valine.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/smooth-scroll@16.1.3/dist/smooth-scroll.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/autocomplete.js@0.37.1/dist/autocomplete.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lunr@2.3.8/lunr.min.js"></script><script type="text/javascript" src="/lib/lunr/lunr.stemmer.support.min.js"></script><script type="text/javascript" src="/lib/lunr/lunr.zh.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lazysizes@5.2.2/lazysizes.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lightgallery.js@1.2.0/dist/js/lightgallery.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lg-thumbnail.js@1.2.0/dist/lg-thumbnail.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lg-zoom.js@1.2.0/dist/lg-zoom.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/clipboard@2.0.6/dist/clipboard.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sharer.js@0.4.0/sharer.min.js"></script><script type="text/javascript">window.config={"code":{"copyTitle":"复制到剪贴板","maxShownLines":10},"comment":{"valine":{"appId":"NEWqsYTB88Sn8Pq0AsYDaXcn-gzGzoHsz","appKey":"bvPBG3pirvLqtjU6IYVxGIyP","avatar":"mp","el":"#valine","emojiCDN":"https://cdn.jsdelivr.net/npm/emoji-datasource-google@5.0.1/img/google/64/","emojiMaps":{"100":"1f4af.png","alien":"1f47d.png","anger":"1f4a2.png","angry":"1f620.png","anguished":"1f627.png","astonished":"1f632.png","black_heart":"1f5a4.png","blue_heart":"1f499.png","blush":"1f60a.png","bomb":"1f4a3.png","boom":"1f4a5.png","broken_heart":"1f494.png","brown_heart":"1f90e.png","clown_face":"1f921.png","cold_face":"1f976.png","cold_sweat":"1f630.png","confounded":"1f616.png","confused":"1f615.png","cry":"1f622.png","crying_cat_face":"1f63f.png","cupid":"1f498.png","dash":"1f4a8.png","disappointed":"1f61e.png","disappointed_relieved":"1f625.png","dizzy":"1f4ab.png","dizzy_face":"1f635.png","drooling_face":"1f924.png","exploding_head":"1f92f.png","expressionless":"1f611.png","face_vomiting":"1f92e.png","face_with_cowboy_hat":"1f920.png","face_with_hand_over_mouth":"1f92d.png","face_with_head_bandage":"1f915.png","face_with_monocle":"1f9d0.png","face_with_raised_eyebrow":"1f928.png","face_with_rolling_eyes":"1f644.png","face_with_symbols_on_mouth":"1f92c.png","face_with_thermometer":"1f912.png","fearful":"1f628.png","flushed":"1f633.png","frowning":"1f626.png","ghost":"1f47b.png","gift_heart":"1f49d.png","green_heart":"1f49a.png","grimacing":"1f62c.png","grin":"1f601.png","grinning":"1f600.png","hankey":"1f4a9.png","hear_no_evil":"1f649.png","heart":"2764-fe0f.png","heart_decoration":"1f49f.png","heart_eyes":"1f60d.png","heart_eyes_cat":"1f63b.png","heartbeat":"1f493.png","heartpulse":"1f497.png","heavy_heart_exclamation_mark_ornament":"2763-fe0f.png","hole":"1f573-fe0f.png","hot_face":"1f975.png","hugging_face":"1f917.png","hushed":"1f62f.png","imp":"1f47f.png","innocent":"1f607.png","japanese_goblin":"1f47a.png","japanese_ogre":"1f479.png","joy":"1f602.png","joy_cat":"1f639.png","kiss":"1f48b.png","kissing":"1f617.png","kissing_cat":"1f63d.png","kissing_closed_eyes":"1f61a.png","kissing_heart":"1f618.png","kissing_smiling_eyes":"1f619.png","laughing":"1f606.png","left_speech_bubble":"1f5e8-fe0f.png","love_letter":"1f48c.png","lying_face":"1f925.png","mask":"1f637.png","money_mouth_face":"1f911.png","nauseated_face":"1f922.png","nerd_face":"1f913.png","neutral_face":"1f610.png","no_mouth":"1f636.png","open_mouth":"1f62e.png","orange_heart":"1f9e1.png","partying_face":"1f973.png","pensive":"1f614.png","persevere":"1f623.png","pleading_face":"1f97a.png","pouting_cat":"1f63e.png","purple_heart":"1f49c.png","rage":"1f621.png","relaxed":"263a-fe0f.png","relieved":"1f60c.png","revolving_hearts":"1f49e.png","right_anger_bubble":"1f5ef-fe0f.png","robot_face":"1f916.png","rolling_on_the_floor_laughing":"1f923.png","scream":"1f631.png","scream_cat":"1f640.png","see_no_evil":"1f648.png","shushing_face":"1f92b.png","skull":"1f480.png","skull_and_crossbones":"2620-fe0f.png","sleeping":"1f634.png","sleepy":"1f62a.png","slightly_frowning_face":"1f641.png","slightly_smiling_face":"1f642.png","smile":"1f604.png","smile_cat":"1f638.png","smiley":"1f603.png","smiley_cat":"1f63a.png","smiling_face_with_3_hearts":"1f970.png","smiling_imp":"1f608.png","smirk":"1f60f.png","smirk_cat":"1f63c.png","sneezing_face":"1f927.png","sob":"1f62d.png","space_invader":"1f47e.png","sparkling_heart":"1f496.png","speak_no_evil":"1f64a.png","speech_balloon":"1f4ac.png","star-struck":"1f929.png","stuck_out_tongue":"1f61b.png","stuck_out_tongue_closed_eyes":"1f61d.png","stuck_out_tongue_winking_eye":"1f61c.png","sunglasses":"1f60e.png","sweat":"1f613.png","sweat_drops":"1f4a6.png","sweat_smile":"1f605.png","thinking_face":"1f914.png","thought_balloon":"1f4ad.png","tired_face":"1f62b.png","triumph":"1f624.png","two_hearts":"1f495.png","unamused":"1f612.png","upside_down_face":"1f643.png","weary":"1f629.png","white_frowning_face":"2639-fe0f.png","white_heart":"1f90d.png","wink":"1f609.png","woozy_face":"1f974.png","worried":"1f61f.png","yawning_face":"1f971.png","yellow_heart":"1f49b.png","yum":"1f60b.png","zany_face":"1f92a.png","zipper_mouth_face":"1f910.png","zzz":"1f4a4.png"},"enableQQ":false,"highlight":true,"lang":"zh-cn","pageSize":10,"placeholder":"你的评论 ...","recordIP":true,"serverURLs":"https://newqsytb.lc-cn-n1-shared.com","visitor":true}},"lightGallery":{"actualSize":false,"exThumbImage":"data-thumbnail","hideBarsDelay":2000,"selector":".lightgallery","speed":400,"thumbContHeight":80,"thumbWidth":80,"thumbnail":true},"search":{"highlightTag":"em","lunrIndexURL":"/index.json","lunrLanguageCode":"zh","lunrSegmentitURL":"/lib/lunr/lunr.segmentit.js","maxResultLength":10,"noResultsFound":"没有找到结果","snippetLength":50,"type":"lunr"}};</script><script type="text/javascript" src="/js/theme.min.js"></script></body>
</html>
